package storage_spec::drbd8;

BEGIN {
        if(!exists($::ENV{TRUECL_CODE})) {
                die("Missing TRUECL_CODE environment variable.");
        }
        if(!exists($::ENV{TRUECL_LOG})) {
                die("Missing TRUECL_LOG environment variable.");
        }
        if(!exists($::ENV{TRUECL_CFG})) {
                die("Missing TRUECL_CFG environment variable.");
        }
	push @::INC,"$::ENV{TRUECL_CODE}/lib";
}

use clreq;
use cldb;
use Data::Dumper;
use Sys::Hostname;
use lvm;
use common_requests;
use nethardware;
use Config;
use clutils qw(DFPORT get_nodes_in_cluster);

#########################################################################
# Returns reference to hash if the specified drbd device exists,	#
# otherwise undef. If returns the following fields;			#
# cs (Connection State): Unconfigured, Standalone, Connected,		#
# SyncSource,SyncTarget, PausedSync (lower case)			#
# st (Local/Remote): Primary|Secondary|Unknown/Primary|Secondary|Unknown#
# ld (Local data consistancy): consistent, unconsistent			#
# good, bad, total							#
# est, speed								#
#########################################################################

sub _map_select_output($) {
my $r=$_[0];

	my %results=();
	my $chost;
	for $chost (keys(%$r)) {
		if(!exists($results{$chost})) {
			$results{$chost}=[];
		}
		my $l=$r->{$chost};
		if($l =~ /^(ERROR|WARNING)/) {
			$results{$chost}=[$l];
			next;
		}
		my @LI=split(/\n/,$l);
		my @f=split(/\|/,$LI[0]);
		my $cl;
		for $cl (@LI[1 .. $#LI]) {
			my %fields=();
			my $item=0;
			my @citem=split(/\|/,$cl);
			while($item<@citem) {
				$fields{$f[$item]}=$citem[$item];
				$item++;
			}
			push @{$results{$chost}},{%fields};
		}
	}
	return {%results};
}

sub get_drbd_dev_status {
my $devnum=$_[0];
my ($fd,@f,@f2,@f3,@f4,$cd,$cline);
my $info={};
my $drbd_version;

	@f=();
	@f2=();
	@f3=();
	@f4=();
	open($fd,"/proc/drbd") or return(undef);
	$_=<$fd>; @f=split; $drbd_version=$f[1]; @f=(); <$fd>; # Ignore first two lines - but get version!
	while(<$fd>) {
		chomp;
		if(/^\s*([0-9]+):\s+/) {
			$cd=$1;
			s/^\s+//;
			@f=split(/\s+/,$_);
			if($cd != $devnum) {
				if($f[1] eq "cs:Unconfigured") {
					next;
				}
				<$fd>;
				next;
			}
			if($f[1] ne "cs:Unconfigured") {
				@f2=(); @f3=(); @f4=();
				$cline=<$fd>; last if !defined($cline);
				$cline =~ s/^\s+//; last if !defined($cline);
				@f2=split(/\s+/,$cline);
				$cline=<$fd>; last if !defined($cline);
				$cline =~ s/^\s+//;
				if($cline =~ /\[/) {
					@f3=split(/\s+/,$cline);
					$cline=<$fd>;
					$cline =~ s/^\s+//;
					@f4=split(/\s+/,$cline);
				}
			}
			last;
		}
	}
	close($fd);
	if(scalar(@f2)) {
		foreach (@f,@f2) {
			next if ! /([a-z]+):(.*)/;
			$info->{$1}=lc($2);
		}
		if(scalar(@f3)) {
			my ($x,$b1,$b2);
			my $multi=1;
			$x=$f3[$#f3];
			if($x =~ /\)M$/) {
				$multi=1024;
			}
			$x =~ s/\(//;
			$x =~ s/\)[MK]//;
			($b1,$b2)=split(/\//,$x);
			if($drbd_version =~ /^8\./) {
				$info->{total}=$b2*$multi;
				$info->{bad}=$b1*$multi;
				$info->{good}=$info->{total}-$info->{bad};
			} else {
				$info->{total}=$b2*$multi;
				$info->{good}=$b1*$multi;
				$info->{bad}=$info->{total}-$info->{good};
			}
		}
		if(scalar(@f4)) {
			my ($hours,$mins,$secs);
			
			($hours,$mins,$secs)=split(/:/,$f4[1]);
			$info->{est}=sprintf("%.1f",($hours*60+$mins)+($secs/60));
			$f4[3] =~ s/,//g;
			$info->{speed}=$f4[3];
		}
		#################################################################
		# For drbd v8 we use {ds} rather than {ld} so convert...	#
		#################################################################
		if(!exists($info->{ld}) && exists($info->{ds})) {
			if($info->{ds} =~ /^uptodate/) {
				$info->{ld}="consistent";
			} else {
				$info->{ld}="inconsistent";
			}
		}
		return $info;
	}
	return undef;
}

sub replicated {
	return 1;
}

sub summary_sync_state {
my $ref=$_[0];
my ($cs,$st,$ld);

	$cs=$ref->{cs};
	$st=$ref->{st} if exists($ref->{st});
	$ld=$ref->{ld} if exists($ref->{st});

	if(!defined($st) || !defined($ld)) {
		return "unknown";
	}
	return "sync" if $cs eq "connected";
	if(scalar(grep {/^$cs$/} qw(syncsource synctarget pausedsynct pausedsyncs))) {
		return "syncing";
	}
	return "unsync";
}

sub summary_valid_data {
my $ref=$_[0];
my ($x,$l,$r);

	if(!exists($ref->{ld})) {
		return "unknown";
	}
	if($ref->{ld} eq "consistent" && $ref->{cs} eq "connected") {
		return "both";
	}
	if($ref->{ld} eq "consistent") {
		return "local";
	}
	$x=$ref->{cs};
	#################################################################
	# Check to see if a sync is occuring, since this defines	#
	# were the valid data source is.				#
	#################################################################
	if($x eq "syncsource" || $x eq "pausedsyncs") {
		return "local";
	}
	if($x eq "synctarget" || $x eq "pausedsynct") {
		return "remote";
	}

	return "none!";
}

sub summary_connection_state {
my $ref=$_[0];
my ($st,$cs);

	$st=$ref->{st};
	$cs=$ref->{cs};
	if($st =~ /primary/ && $cs =~ /wfconnection|syncsource|synctarget|connected/) {
		return "active";
	}
	if($cs =~ /standalone/) {
		return "standalone";
	}
	return "inactive";
}
my $error_msg=undef;

#########################################################################
# We output debug if --debug was on the command line, or a local	#
# debug flag file exists TRUECL_CODE/flags/clreqd.debug.		#
#########################################################################

sub dprint {
	if(-f "$::ENV{TRUECL_CODE}/flags/clreqd.debug") {
		::msg("Debug: ",join("",@_));
		return;
	}
	return if !defined($::DEBUG);
	return if $::DEBUG==0;
	::msg("Debug: ",join("",@_));
}

sub get_error_msg {
	return $error_msg;
}

sub type {
my $self=shift;
	return $self->{TYPE};
}


#########################################################################
# The storage spec stuff for drdb must worry about port allocations.	#
#########################################################################

sub get_drbd_proc_version {
	return undef if ! -f "/proc/drbd";
	my $fd;
	open($fd,"/proc/drbd") or return(undef);
	my $l=<$fd>;
	my ($x) = ($l =~ /^Version:\s+(\d+\.\d+\.\d+)\s+/i);
	close($fd);
	return $x;
}

sub new {
my $proto=shift;
my $class=ref($proto) || $proto;
my $self={};

	if($Config{osname} ne "linux") {
		$self->{TYPE}="Replicated DRBD 8.2.x Image";
		bless($self,$class);
		return $self;
	}
	my $max_drbd8_resources=64;
	my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($db)) {
		$error_msg="Unable to open config local database connection.";
		return undef;
	}
	my $href=$db->get_rows("SELECT value from GLOBALS WHERE var = 'max_drbd8_devs';");
	if(exists($href->[0]->{value}) && defined($href->[0]->{value})) {
		$max_drbd8_resources=int($href->[0]->{value});
		$max_drbd8_resources=4 if $max_drbd8_resources<4;
	}
	$db->disconnect;

	#################################################################
	# For drbd 8.2.x we move drbdadm out of the way, since it	#
	# is called implicitly under certain conditions which is not	#
	# something we want!						#
	#################################################################

	if(-f "/sbin/drbdadm") {
		unlink("/sbin/drbdadm.moved-by-truecl");
		rename("/sbin/drbdadm","/sbin/drbdadm.moved-by-truecl");
	}

	#################################################################
	# The above will indicate how many resources to configure if	#
	# the module is to be loaded.					#
	#################################################################

	my $drbd_meta="/sbin/drbdmeta";
	my $drbd_setup="/sbin/drbdsetup";
	my $drv=get_drbd_proc_version();
	if(!defined($drv)) {
		my $r=`/sbin/modprobe drbd minor_count=$max_drbd8_resources 2>&1`;
		if(length($r)) {
			$r="Unable to run /sbin/modprobe!" if !defined($r);
			$error_msg="Unable to load DRBD module:\n$r";
			return undef;
		}
		$drv=get_drbd_proc_version();
	}
	if(defined($drv) && -x "$::ENV{TRUECL_CODE}/drbd-bin/drbdsetup-$drv") {
		$drbd_setup="$::ENV{TRUECL_CODE}/drbd-bin/drbdsetup-$drv";
	}
	$self->{drbd_setup}=$drbd_setup;
	if(defined($drv) && -x "$::ENV{TRUECL_CODE}/drbd-bin/drbdmeta-$drv") {
		$drbd_meta="$::ENV{TRUECL_CODE}/drbd-bin/drbdmeta-$drv";
	}
	$self->{drbd_meta}=$drbd_meta;
	#################################################################
	# Attempt to attach to config db and create drbd resources	#
	# table if necessary.						#
	#################################################################

	my $cldb=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($cldb)) {
		$error_msg="Unable to connect to 'config' database.\nReason: " . (cldb::error)[1];
		return undef;
	}
	my $r=$cldb->exists("table","storage_drbd8");
	if($r==0) {
		if(! $cldb->do("CREATE TABLE storage_drbd8(devnum INTEGER, appname VARCHAR(50), syncrate INTEGER, devport VARCHAR(20), vglv VARCHAR(20), topology VARCHAR(20), vischeck VARCHAR(200), synched INTEGER, validhost VARCHAR(20));")) {
			my ($e1,$e2)=cldb::error();
			$error_msg="Unable to create storage_drbd8 table. [rc=$e1,msg=$e2]";
			return undef;
		}
	}
	$cldb->disconnect;
	$self->{TYPE}="Replicated DRBD 8.2.x Image";
	bless($self,$class);
	return $self;
}

#########################################################################
# The minimum number of nodes to run on - would it work on 1 only??	#
#########################################################################

sub min_nodes {
	return 1;
}

#########################################################################
# Can used a single image on any of up to 2 nodes - replication.	#
#########################################################################

sub max_nodes {
	return 2;
}

#########################################################################
# Can multiple nodes run the application simultaneously? Although	#
# supported on drbd 8.2.x we do not support it in truecl.		#
#########################################################################

sub allow_concurrent {
	return 0;
}

sub repl_ip {
my $self=shift;

	my $cldb=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($cldb)) {
		return(0,"Unable to connect to 'config' database.\nReason: " . (cldb::error)[1]);
	}
	my $ref=$cldb->get_rows("SELECT * FROM storage_drbd8;");
	my $ref2=$cldb->get_rows("SELECT * FROM topology;");
	if(!defined($ref)) {
		return(0,"No information returned from query of storage_drbd8.\nReason: ". (cldb::error)[1]);
	}
	my %tops=();
	for my $cref (@$ref) {
		if(defined($cref->{topology})) {
			$tops{$cref->{topology}}={};
		}
	}
	#################################################################
	# We have a list of topologies used by DRBD, so now we		#
	# need to get list of cards for each topology.			#
	#################################################################
	for my $cref (@$ref2) {
		if(defined($cref->{name}) && defined($cref->{lcards})) {
			if(exists($tops{$cref->{name}})) {
				$tops{$cref->{name}}->{lcards}=$cref->{lcards};
			}
		}
	}
	my $nethw=nethardware->new();
	for my $ctop (keys %tops) {
		next if !exists($tops{$ctop}->{lcards});
		my $l=$tops{$ctop}->{lcards};
		for my $ccard (split(/,/,$l)) {
			my $rr=$nethw->cardref($ccard);
			next if !defined $rr;
			if(exists($rr->{IPv4}) && defined($rr->{IPv4}) &&
				$rr->{IPv4} ne "") {
				$tops{$ctop}->{IPv4}=$rr->{IPv4};
			}
		}
	}
	#################################################################
	# We have all the information needed, so return it in the	#
	# following format:						#
	# topname|ip,...						#
	#################################################################

	my @i=();
	for my $ctop (keys %tops) {
		if(!exists($tops{$ctop}->{IPv4})) {
			push @i,"$ctop=NONE";
		} else {
			push @i,"$ctop=$tops{$ctop}->{IPv4}";
		}
	}
	return (1,join(",",@i));
}

#########################################################################
# The below is called via clreqd - and thus must not use clreqd itself!	#
# This assumes that all file systems and logical volumes in use for the	#
# volume group are closed.
#########################################################################

sub run_before_shutdown_on_current($$;$) {
my ($self,$vgname,$lvname)=@_;

	#################################################################
	# For drbd8 we connect to the database to get details of	#
	# devices that we need to clear down.				#
	#################################################################

	my $cldb=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($cldb)) {
		return(0,"Unable to connect to 'config' database.\nReason: " . (cldb::error)[1]);
	}
	my $r=$cldb->exists("table","storage_drbd8");
	return(1,undef) if !$r;

	#################################################################	
	# If the table does not exist, then assume we have no work to	#
	# do.								#
	#################################################################	
 
	my $ref;
	if(defined($lvname)) {
		$ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv = '$vgname/$lvname';");
	} else {
		$ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv LIKE '$vgname/%';");
	}

	my $cref;
	for $cref (@$ref) {
		next if !defined($cref);
		#########################################################
		# For each lv/vg disconnect and down the device...	#
		#########################################################
		$cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} down";
		my $rc=`$cmd 2>&1 >/dev/null`;
		chomp $rc;
		return(0,"Unable to 'down' device $cref->{devnum}. Error: $rc") if length($rc);
	}
	return(1,undef);
}

#########################################################################
# The below is called via clreqd - and thus must not use clreqd itself!	#
# So each host will get this called via a storage_spec call for a 	#
# volume group / application.						#
#########################################################################

sub run_before_shutdown_on_non_current($$$;$) {
my ($self,$vgname,$repl_map,$lvname)=@_;

	if(defined($repl_map)) {
		my @repl_split=split(/\s+/,$repl_map);
		for my $cent (@repl_split) {
			my ($node,$info)=($cent =~ /^(.*):(.*)/);
			$node_repl{$node}={};
			my @x=split(/,/,$info);
			for my $cent2 (@x) {
				my ($top,$ip)=($cent2 =~ /^(.*)=(.*)/);
				$node_repl{$node}->{$top}=$ip;
			}
		}
	} else {
		$repl_map="UNDEF";
	}
	#################################################################
	# For DRBD we also issue the disconnect / down statements - 	#
	# though here the return codes are ignored [since the devices	#
	# might not be up] ...						#
	#################################################################

	if(!defined($lvname)) {
		dprint "run_before_shutdown_on_non_current: args=$vgname,$repl_map,lvname=undef";
	} else {
		dprint "run_before_shutdown_on_non_current: args=$vgname,$repl_map,lvname=$lvname";
	}
	my $cldb=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($cldb)) {
		return(0,"Unable to connect to 'config' database.\nReason: " . (cldb::error)[1]);
	}
	my $r=$cldb->exists("table","storage_drbd8");
	return(1,undef) if !$r;

	my $ref;
	if(defined($lvname)) {
		$ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv = '$vgname/$lvname';");
	} else {
		$ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv LIKE '$vgname/%';");
	}
	my $cref;
	my $stop_again=1;
	for $cref (@$ref) {
		next if !defined($cref);
		my $r=get_drbd_dev_status($cref->{devnum});
		if(!defined($r)) {
			return(0,"Status information for device $cref->{devnum} not found in '/proc/drbd'.");
		}
		$cs=lc($r->{cs});
		#########################################################
		# If this node is the source of the current data then	#
		# we can not shutdown such resources first without	#
		# causing a remote panic - hence we ignore devices	#
		# in this state and return 2 at the end = indicating	#
		# this should be called _again_ after stopping on	#
		# the current node.					#
		#########################################################
		if($cs eq "syncsource") {
			$stop_again=2;
			dprint "Device $cref->{devnum} must stop remotely first.";
			next;
		}
		#########################################################
		# For each lv/vg down the device...			#
		#########################################################
		$cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} down";
		dprint "cmd=$cmd\n";
		my $rc=`$cmd 2>&1 >/dev/null`;
		chomp $rc;
		return(0,"Unable to 'down' device $cref->{devnum}. Error: $rc") if length($rc);
	}
	return($stop_again,undef);
}

sub _run_before_start_on_current {
my ($self,$vgname,$allow_stale,$non_current,$force,$lvname)=@_;
my $lvm=new lvm;
my %node_repl=();

	#################################################################
	# If a REPL_MAP is available then load in the details of all	#
	# topologies to save time [or deadlocks] making further		#
	# queries.							#
	#################################################################

	if(defined($::REPL_MAP)) {
		my @repl_split=split(/\s+/,$::REPL_MAP);
		for my $cent (@repl_split) {
			my ($node,$info)=($cent =~ /^(.*):(.*)/);
			$node_repl{$node}={};
			my @x=split(/,/,$info);
			for my $cent2 (@x) {
				my ($top,$ip)=($cent2 =~ /^(.*)=(.*)/);
				$node_repl{$node}->{$top}=$ip;
			}
		}
	}

	$force=0 if !defined($force);
	if(!defined($lvname)) {
		$_[5]="?undefined?";
	}
	dprint "_run_before_start_on_current: args=",join(",",@_);
	if(defined($non_current)) {
		$non_current=1 if $non_current eq "NON_CURRENT";
		$non_current=2 if $non_current eq "NON_CURRENT_AFTER";
	} else {
		$non_current=0;
	}

	#################################################################
	# Attempt activate - might not be necessary - but worth		#
	# doing just in case!						#
	#################################################################

	my ($rc,$r)=$lvm->vg_activate(VGNAME=>$vgname,FORCE=>$force);

	#################################################################
	# For DRBD we also issue the disconnect / down statements - 	#
	# though here the return codes are ignored [since the devices	#
	# might not be up] ...						#
	#################################################################

	my $cldb=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($cldb)) {
		return(0,"Unable to connect to 'config' database.\nReason: " . (cldb::error)[1]);
	}
	$r=$cldb->exists("table","storage_drbd8");
	return(1,undef) if !$r;

	#################################################################
	# Before doing anything we need to identify which application	#
	# we are running... and which node acts as secondary.		#
	#################################################################

	my $ref2=$cldb->get_rows("SELECT * FROM applications WHERE vgs LIKE '$vgname' OR vgs LIKE '$vgname,%' OR vgs LIKE '%,$vgname,%' OR vgs LIKE '$vgname,%';");
	if(@$ref2>1) {
		return(0,"Matched multiple applications using VG '$vgname'!");
	}
	if(!exists($ref2->[0]->{name})) {
		return(0,"Matched no applications using VG '$vgname'.");
	}

	my $appname=$ref2->[0]->{name};
	my $hostname=hostname;

	if(!defined($ref2->[0]->{nodes})) {
		return(0,"Nodes not defined for application '$appname'.");
	}
	my @N=split(/,/,$ref2->[0]->{nodes});
	if(@N!=2) {
		return(0,"DRBD8 based application ['$appname'] must define two nodes.");
	}
	my $othernode=undef;
	if($N[0] eq $N[1]) {
		return(0,"Two different nodes must be specified for application '$appname'.");
	}
	if($N[0] eq $hostname) {
		$othernode=$N[1];
	}
	if($N[1] eq $hostname) {
		$othernode=$N[0];
	}
	if(!defined($othernode)) {
		return(0,"List of nodes for application '$appname' must include current node!");
	}
	
	dprint "node=$hostname,othernode=$othernode\n";
	#################################################################
	# Which IP address do we use for DRBD for the other node - get	#
	# all topology details for the other node firstly...		#
	#################################################################

	my $topology="??????????";
	my $ip="??????????";
	my $ref;
	if(defined($lvname)) {
		$ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv = '$vgname/$lvname';");
	} else {
		$ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv LIKE '$vgname/%';");
	}
	my $cref;

	#################################################################
	# In this section below we can check to see if REPL_MAP is	#
	# set rather than attempting a remote query. Use that info	#
	# if possible - saves lvm_attempt_use from deadlock. 		#
	#################################################################

	my $clreqdxml;
	my $reqd;
	if(!exists($node_repl{$othernode})) {
		$clreqdxml=new clreqdxml("$::ENV{TRUECL_CFG}/clreqd.xml");
		if(! defined($clreqdxml)) {
			return (0,"Unable to load Request Daemon configuration info.", "Reason: " . clreqdxml::error());
		}
		$reqd=new clreq(hosts => [$othernode], clreqdxml => $clreqdxml, timeout => 5);
	} else {
		::msg(Dumper(%node_repl));
	}

	for $cref (@$ref) {
		my ($ctopology,$cip);
		my ($local_cip);
		next if !defined($cref);
		
		if(!defined($cref->{topology}) || $cref->{topology} eq "") {
			return(0,"Toplogy not defined for application/node/vg/lv - '$appname/$hostname/$cref->{vglv}'.");
		}

		::msg("H:top=$cref->{topology}");
		::msg(Dumper(%node_repl));
		if(exists($node_repl{$othernode}->{$cref->{topology}})) {
			$cip=$node_repl{$othernode}->{$cref->{topology}};
			$ctopology=$cref->{topology};
			$ip=$cip;
			$topology=$ctopology;
			$local_cip=$node_repl{$hostname}->{$topology};
		} elsif($cref->{topology} eq $topology) {
			$cip=$ip;
			$ctopology=$topology;
		} else {
			#################################################
			# Must communicate with other node, get the IP 	#
			# used for this topology ...			#
			#################################################
			my $results=$reqd->run(cmd => ["get_topology_ip_addr","TOPOLOGY",$cref->{topology}]);
			if(!defined($results) || !defined($results->{$othernode})) {
				return(0,"No results return for topology query on '$othernode'.");
			}
			chomp($results->{$othernode});
			my @F=split(/\s+/,$results->{$othernode});
			if($F[0] ne "OK") {
				if($force==0) {
					return(0,"Error return from topology query on '$othernode'.\n" . $results->{$othernode});
				}
				::warning("Unable to contact '$othernode' - forced to continue.");
			} else {
				$cip=$F[1];
				$ctopology=$cref->{topology};
			}

			#################################################
			# Also get the local IP address for this 	#
			# network.					#
			#################################################

			if(defined($cip)) {
				my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
				if(!defined($db)) {
					return(0,"Unable to open config local database connection.");
				}
				if(! $db->exists("table","topology")) {
					return(0,"Local config database missing 'topology' table.");
				}
				my $href=$db->get_rows("SELECT * from TOPOLOGY WHERE name = '$ctopology';");
				if(!exists($href->[0]->{name}) || !defined($href->[0]->{name})) {
					return(0,"Topology '$ctoplogy' not defined on '$hostname'.");
				}
				my $cards=$href->[0]->{lcards};
				my $nethw=new nethardware();
				my $ccard;
				$local_cip=undef;
				for $ccard (split(/,/,$cards)) {
					my $rr=$nethw->cardref($ccard);
					next if !defined $rr;
					if(exists($rr->{IPv4}) && defined($rr->{IPv4}) &&
						$rr->{IPv4} ne "") {
						$local_cip=$rr->{IPv4};
						last;
					}
				}
				if(!defined($local_cip)) {
					return(0,"Network cards in topology '$ctopology' on '$hostname' do not define an IP address!");
				}
				# $local_cip=$href->[0]->{name}
			}
		}
		if(defined($cip)) {
			::msg("local_cip=$local_cip,cip=$cip,ctopology=$ctopology");
			dprint "local_cip=$local_cip\n";
		}
		#########################################################
		# For each lv/vg we know have the IP address to 	#
		# connect to, and the port and device number, so run	#
		# the commands.						#
		#########################################################

		if($non_current!=2) {
			my $cmd2=$self->{drbd_meta} . " --force /dev/drbd$cref->{devnum} v08 /dev/$cref->{vglv}_meta 0 get-gi";
			dprint "Running '$cmd2'\n";
			my $str2=`$cmd2 2>&1`;
			my $rc2=$?>>8;
			chomp $str2;
			#################################################
			# If we do not get a result it indicates that	#
			# the meta information is not yet created, so	#
			# do so now...					#
			#################################################
			if(!length($str2)) {
				my $cmd3=$self->{drbd_meta} . " --force /dev/drbd$cref->{devnum} v08 /dev/$cref->{vglv}_meta 0 create-md";
				dprint "Running '$cmd3'\n";
				my $str3=`$cmd3 2>&1`;
				my $rc3=$?>>8;
				if($rc3!=0 && $str3 !~ /New drbd meta data block sucessfully created/) {
					chomp $str3;
					return(0,"Unable to setup meta device information for /dev/$cref->{vglv}. Command:\n$cmd3\nOutput:\n$str3");
				}
			}
			my $cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} down";
			system("$cmd >/dev/null 2>/dev/null");
			$cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} disk /dev/$cref->{vglv} /dev/$cref->{vglv}_meta 0 --create-device";
			dprint "Running '$cmd'\n";
			my $str=`$cmd 2>&1`;
			chomp $str;
			# If drbd8 complains about meta data- set it up now...
			if($str =~ /drbdadm create-md res/) {
				my $cmd3=$self->{drbd_meta} . " --force /dev/drbd$cref->{devnum} v08 /dev/$cref->{vglv}_meta 0 create-md";
				dprint "Running '$cmd3'\n";
				$str=`$cmd3 2>&1`;
				$cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} disk /dev/$cref->{vglv} /dev/$cref->{vglv}_meta 0 --create-device";
				dprint "Running '$cmd'\n";
				$str=`$cmd 2>&1`;
				chomp $str;
			}
			if(length($str)) {
				return(0,"Unable to setup disk for /dev/$cref->{vglv}. Command:\n$cmd\nOutput:\n$str");
			}

			#########################################################
			# Now set up the net details ... if $cip available	#
			#########################################################
			if(defined($cip)) {
				my $devport=$cref->{devport};
				$cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} net $local_cip:$devport $cip:$devport C -m";
				dprint "Running '$cmd'\n";
				$str=`$cmd 2>&1`;
				chomp $str;
				$rc=$?>>8;
				if($rc && length($str)) {
					return(0,"Unable to setup net for /dev/$cref->{vglv}. Command:\n$cmd\nOutput:\n$str");
				}
			} else {
				::warning("No networking details configured.");
			}

			if($non_current==1) {
				my $cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} secondary";
				dprint "Running '$cmd'\n";
				my $str=`$cmd 2>&1`;
				chomp $str;
				my $rc=$?>>8;
				if($rc && length($str)) {
					return(0,"Unable to set as secondary for /dev/$cref->{vglv}. Command:\n$cmd\nOutput:\n$str");
				}
			}

			#########################################################
			# Now set device as primary if possible - if running	#
			# on the current host.					#
			#########################################################
			if($non_current==0) {
				#################################################
				# We need to wait for a connected state before	#
				# we can continue.. or return an error if	#
				# we consider both inconsistent.		#
				#################################################
				my $tot_time=0;
				my $cs="";
				while(1) {
					my $r=get_drbd_dev_status($cref->{devnum});
					if(!defined($r)) {
						return(0,"Status information for device $cref->{devnum} not found in '/proc/drbd'.");
					}
					$cs=lc($r->{cs});
					dprint "dump of r is",Dumper($r);
					if($cs eq "connected" || $cs eq "synctarget" || $cs eq "syncsource") {
						last;
					}
					select(undef,undef,undef,0.10);
					$tot_time+=0.10;
					if($tot_time>25) {
						return(0,"Unable to move to connected state in 5 seconds!");
					}
				}
				#################################################
				# Set as primary if we think the other node is	#
				# invalid.					#
				#################################################
				my $flgs="";
				my $force_primary=0;
				if(defined($cref->{synched}) && $cref->{synched} eq "N") {
					my $cmd2;
					if(defined($cref->{validhost}) && $cref->{validhost} eq $hostname) {
						$cmd2=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} invalidate-remote";
						$force_primary=1;
					} else {
						$cmd2=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} invalidate";
					}
					dprint "Running '$cmd2'\n";
					my $str2=`$cmd2 2>&1`;
					chomp $str2;
					$rc=$?>>8;
					if($rc && length($str2) && $force_primary==0) {
						return(0,"Unable to invalidate stale copy for /dev/$cref->{vglv} on '$hostname'. Command:\n$cmd2\nOutput:\n$str2");
					}
					if($allow_stale==1) {
						$force_primary=1;
					}
				}

				if($force_primary) {
					$cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} primary -o";
				} else {
					$cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} primary";
			 	}
				#################################################
				# This is a special value that lvm_attempt_use	#
				# makes use of to ensure that the functionality	#
				# works as expected, without actually 		#
				# attempting to make primary.			#
				#################################################
				if($allow_stale!=2) {
					dprint "Running '$cmd'\n";
					my $str=`$cmd 2>&1`;
					chomp $str;
					$rc=$?>>8;
					if($rc && length($str)) {
						return(0,"Unable to set as primary for /dev/$cref->{vglv} on '$hostname'. Command:\n$cmd\nOutput:\n$str");
					}
					#########################################
					# If the syncrate is defined we also	#
					# set that up now.			#
					#########################################
					if(defined($cref->{syncrate}) && int($cref->{syncrate})>0) {
						$cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} syncer -r $cref->{syncrate}";
						dprint "Running '$cmd'\n";
						my $str=`$cmd 2>&1`;
						chomp $str;
						if(length($str)) {
							::warning("Unable to set sync rate of '$cref->{syncrate}' for '/dev/drbd$cref->{devnum}'.");
						}
					}
				}
			}
		} else {

			#########################################################
			# If we are running on the secondary we wait for a 	#
			# little while to go into connected state - and then	#
			# issue the invalidate if we think we need to.		#
			#########################################################

			my $tot_time=0;
			my $cs="";
			while(1) {
				my $r=get_drbd_dev_status($cref->{devnum});
				if(!defined($r)) {
					return(0,"Status information for device $cref->{devnum} not found in '/proc/drbd'.");
				}
				$cs=lc($r->{cs});
				dprint "dump of r is",Dumper($r);
				if($cs eq "connected" || $cs eq "synctarget" || $cs eq "syncsource") {
					last;
				}
				select(undef,undef,undef,0.10);
				$tot_time+=0.10;
				if($tot_time>5) {
					my $dd=Dumper($r);
					return(0,"Unable to move to connected state in 5 seconds!\nStatus: $dd");
				}
			}

			#########################################################
			# We also need to see if the cref->{synched}=="N" and	#
			# if so we run 'invalidate' on this machine if the	#
			# valid host is _NOT_ this hostname.			#
			#########################################################
			if(defined($cref->{synched}) && $cref->{synched} eq "N" &&
				defined($cref->{validhost}) && $cref->{validhost} ne $hostname) {
				while(1) {
					my $r=get_drbd_dev_status($cref->{devnum});
					::msg("r=" . Dumper($r));
					last if $r->{st} =~ /primary/;
					$cmd=$self->{drbd_setup} . " /dev/drbd$cref->{devnum} invalidate";
					dprint "Running '$cmd'\n";
					my $str=`$cmd 2>&1`;
					chomp $str;
					if($str =~ /Operation now in progress/) {
						select(undef,undef,undef,0.25);
						next;
					}
					$rc=$?>>8;
					if($rc && length($str)) {
						if($str =~ /State change failed:/i) {
							return(1,undef);	
						}
						if($str =~ /Refusing to be inconsistent on both nodes/i) {
							return(1,undef);	
						}
						if($str =~ /resync since it is already active/i) {
							return(1,undef);	
						}
						return(0,"Unable to 'invalidate' /dev/$cref->{vglv} on '$hostname'. Command:\n$cmd\nOutput:\n$str");
					}
				}
			} else {
				dprint "start_after_on_non_current has nothing to do.\n";
			}
		}
	}
	return(1,undef);
}

sub run_before_start_on_current {
my ($self,$vgname,$allow_stale,$force,$lvname)=@_;

	return (_run_before_start_on_current($self,$vgname,$allow_stale,1,$force,$lvname));
}

#########################################################################
# Attempt to bring up the drbd devices in secondary mode.		#
#########################################################################

sub run_before_start_on_non_current($$;$$) {
my ($self,$vgname,$repl_map,$lvname)=@_;

	if(defined($repl_map)) {
		$::REPL_MAP=$repl_map;
	}
	return (_run_before_start_on_current($self,$_[1],0,"NON_CURRENT",0,$lvname));
}

#########################################################################
# We use the main routine, but indicate we are checking for stuff	#
# after it has been started on current. This might invalidate a copy	#
# to force a sync for example...					#
#########################################################################

sub run_after_start_on_non_current {
my ($self,$vgname,$repl_map,$lvname)=@_;

	if(defined($repl_map)) {
		$::REPL_MAP=$repl_map;
	}
	#################################################################
	# REPL_MAP means that IP info is not needed from remote		#
	# machines...							#
	#################################################################
	return (_run_before_start_on_current($self,$_[1],0,"NON_CURRENT_AFTER",0,$lvname));
}

#########################################################################
# This routine is used just before the file systems are mounted. 	#
# Returns:	(0,msg)		Failed.					#
#		(1,undef)	Worked as expected.			#
#########################################################################

sub run_after_start_on_current($$;$) {
my ($self,$vgname,$lvname)=@_;

	my $cldb=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($cldb)) {
		return(0,"Unable to open config local database connection.");
	}
	my $r=$cldb->exists("table","storage_drbd8");
	return(0,"Table 'storage_drbd8' does not exist.") if !$r;

	my $ref2=$cldb->get_rows("SELECT * FROM globals WHERE var = 'def_drbd8_sync';");
	my $rate=1024;
	if(defined($ref2) && defined($ref2->[0]->{value}) &&
		int($ref2->[0]->{value})>=10) {
		$rate=$ref2->[0]->{value};
	}
	my $ref;
	if(defined($lvname)) {
		$ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv = '$vgname/$lvname';");
	} else {
		$ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv LIKE '$vgname/%';");
	}
	$cldb->disconnect();
	if(!defined($ref) || !defined($ref->[0]->{devnum})) {
		# Unexpected error
		return(0,"Unable to find required storage_drbd8 record information!");
	}
	my $drbd_setup=$self->{drbd_setup};
	for my $crec (@$ref) {
		next if !defined($crec->{devnum});
		my $devnum=$crec->{devnum};
		my $vglv=$crec->{vglv};
		#########################################################
		# We set the rate if no per-device rate was available	#
		#########################################################
		my $cmd;
		if(!defined($crec->{syncrate})) {
			$cmd=$self->{drbd_setup} . " /dev/drbd$devnum syncer -r $rate";
		} else {
			$cmd=$self->{drbd_setup} . " /dev/drbd$devnum syncer -r $crec->{syncrate}";
		}
		my $rc=`$cmd 2>&1 >/dev/null`;
		chomp $rc;
		return(0,"Unable to set rate for '$vglv' to '$rate' for device '/dev/drbd$devnum'. Error: $rc") if length($rc);
		$cmd=$self->{drbd_setup} . " /dev/drbd$devnum primary";
		$rc=`$cmd 2>&1 >/dev/null`;
		if($rc =~ /Refusing to be Primary without at least one UpToDate disk|State change failed:/i) {
			$cmd=$self->{drbd_setup} . " /dev/drbd$devnum primary -o";
			$rc=`$cmd 2>&1 >/dev/null`;
		}
		chomp $rc;
		my $h=hostname;
		return(0,"Unable to set primary for '$vglv' for device '/dev/drbd$devnum' on '$h'. Error: $rc") if length($rc);
	}
	return(1,undef);
}

#########################################################################
# Will return (1,undef) - if resources allocated, or (0,errmsg) if not. #
# Although this is called for both nodes, only one of the nodes 	#
# actually does the work! This is to prevent them both attempting to	#
# allocate resources and get a mismatch.				#
#									#
# We also need to check the vg/lv exist on both nodes and are the	#
# same size, and create _meta if not already existing.			#
# The extras must include a 'topology' setting, but for drbd8 it can	#
# also include syncrate,auth_crc,data_crc optionally.			#
#									#
#########################################################################

sub alloc_resources_extras {
	return ('topology');
}

sub alloc_resources($$$$$$;$) {
my ($self,$vgname,$lvname,$appname,$nodelist,$openlocally,%extras)=@_;

	my @N=split(/,/,$nodelist);
	if(@N!=2) {
		return(0,"DRBD8 storage types require exactly two nodes for configuration.");
	}
	if(!exists($extras{topology})) {
		return(0,"DRBD8 storage types require 'topology' to be specified.");
	}

	#################################################################
	# Get a free DRBD item from the table on both nodes ... using	#
	# a clreqd request.						#
	#################################################################
	my $clreqdxml=new clreqdxml("$::ENV{TRUECL_CFG}/clreqd.xml");
	if(! defined($clreqdxml)) {
		return (0,"Unable to load Request Daemon configuration info.", "Reason: " . clreqdxml::error());
	}
	my $reqd=new clreq(hosts => [@N], clreqdxml => $clreqdxml, timeout => 10);

	my @N2=sort(@N);
	my $other_node;
	if($N2[0] eq hostname) {
		$other_node=$N2[1];
	} else {
		$other_node=$N2[0];
	}
	#################################################################
	# If resources are already allocated then abort with an errror.	#
	# Changing of nodes for DRBD8 allocated storage is not yet	#
	# supported.							#
	#################################################################

	my $sql="SELECT * FROM storage_drbd8 WHERE appname = '$appname' AND vglv = '$vgname/$lvname';";
	$results=$reqd->run(cmd => ["perform_adhoc_sql","SQL",$sql,"DBTYPE","config"]);
	if(!defined($results)) {
		return(0,"No results returned from perform_adhoc_sql request.");
	}
	my $r=_map_select_output($results);
	my $rec;
	my $found=0;
	for $rec (keys %$r) {
		next if ! ref($r->{$rec}->[0]);
		if($r->{$rec}->[0]->{appname}) {
			$found=1;
		}
	}
	if($found) {
		return(0,"Changing nodes/reallocating resources for DRBD8 storage is not yet supported.");
	}

	#################################################################
	# We need to allocate a DRBD device, 128 MB _meta volume 	#
	# [if it does not exist], and a port.				#
	# If the globals does not contain a max_drbd8_devs variable	#
	# we assume 64 devices can be loaded.				#
	#################################################################

	my $max_drbd8_resources=64;
	my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($db)) {
		return(0,"Unable to open config local database connection.");
	}
	my $syncrate=1024;
	if(exists($extras{syncrate})) {
		$syncrate=int($extras{syncrate});
	} else {
		my $href22=$db->get_rows("SELECT value from GLOBALS WHERE var = 'def_drbd8_sync';");
		if(exists($href22->[0]->{value}) && defined($href22->[0]->{value})) {
			$syncrate=int($href22->[0]->{value});
		}
	}
	my $href=$db->get_rows("SELECT value from GLOBALS WHERE var = 'max_drbd8_devs';");
	if(exists($href->[0]->{value}) && defined($href->[0]->{value})) {
		$max_drbd8_resources=int($href->[0]->{value});
		$max_drbd8_resources=4 if $max_drbd8_resources<4;
	}
	$syncrate=256 if $syncrate<256;
	$db->disconnect;

	my $results=$reqd->run(cmd => ["get_free_drbd8_devs"]);
	if(!defined($results)) {
		return(0,"No results return for drbd8 free devs.");
	}
	#################################################################
	# The results are in OK X[-Y],... format. So need to find a	#
	# common one between them ...					#
	#################################################################

	my %free_devs=();
	my $cnode;
	$chosen=0;
	for $cnode (keys(%$results)) {
		my @F=split(/\s+/,$results->{$cnode});
		if($F[0] ne "OK") {
			return(0,"Got result: '$results->{$cnode}' from $cnode.");
		}
		if(!defined($F[1])) {
			return(0,"Node '$cnode' has no free drbd8 devices.");
		}
		my $cent;
		my @CL=();
		for $cent (split(/,/,$F[1])) {
			my ($x,$y)=($cent =~ /^(\d+)-(\d+)/);
			if(!defined($y)) {
				push @CL,$cent;
				next;
			}
			push @CL,$_ for ($x .. $y);
		}
		$free_devs{$cnode}=[@CL];
	}

	#################################################################
	# Both nodes have had lists of drbd devices expanded to full	#
	# lists, so cross-reference.					#
	#################################################################

	my $hostname=hostname;	
	my $cdev; my $usedev=-1;
	for $cdev (@{$free_devs{$other_node}}) {
		if(grep {$cdev == $_} @{$free_devs{$hostname}}) {
			$usedev=$cdev; last;
		}
	}

	if($usedev==-1) {
		return(0,"No common free DRBD devices between '$hostname' and '$other_node'.");
	}

	#################################################################
	# OK usedev == common free dev, so now get a common free port.	#
	#################################################################

	my $common=new common_requests(timeout=>5,parallel=>0,clreqdxml=>$clreqdxml,force=>1);
	if(!defined($common)) {
		return(0,"Unable to initialise 'common requests' object!");
	}

	my ($list,$err)=$common->get_common_free_port(count=>1,force=>1);
	if(!defined($list)) {
		return(0,"No results returned from 'get_common_free_port' call.\nError: $err");
	}
	my $inet_port=$list->[0];

	#################################################################
	# So now we have usedev==dev number, inet_port = free port, so	#
	# now attempt to make the _meta lv if it does not already	#
	# exist.							#
	# Again use the clreqd requests to allow handling of local/	#
	# remote processing for both nodes here.			#
	#################################################################

	$results=$reqd->run(cmd => ["lvm_lvlist","VGNAME",$vgname,"REFRESH",1]);
	if(!defined($results)) {
		return(0,"No results return for lvm_lvlist request.");
	}
	my $volsize=undef;
	for $cnode ($other_node,$hostname) {
		my @RR=split(/\n/,$results->{$cnode});
		my @RR2=grep {/^$lvname\s+/} @RR;
		if(@RR2==0) {
			return(0,"Volume '$lvname' does not exist on $cnode!");
		}
		my @F=split(/\s+/,$RR2[0]);
		$volsize=$F[1] if !defined($volsize);
		if($F[1] != $volsize) {
			return(0,"Volume size differ on both nodes [$volsize compared to $F[1]]");
		}
	}

	for $cnode ($other_node,$hostname) {
		my @RR=split(/\n/,$results->{$other_node});
		my @RR2=grep {/^${lvname}_meta\s+/} @RR;
		if(@RR2) {
			my @F=split(/\s+/,$RR2[0]);
			if(int($F[1]) < 131072) {
				return(0,"Size of meta volume for '$vgname/$lvname' is too small - it is '$F[1]' - but should be at least 131072.");
			}
		} else {
			#################################################
			# The lv does not exist, so need to create it.	#
			#################################################
			my $reqd2=new clreq(hosts => [$cnode], clreqdxml => $clreqdxml, timeout => 5);
			my $res2=$reqd2->run(cmd => ["lvm_lvcreate","VGNAME",$vgname,"LVNAME","${lvname}_meta","SIZE",128*1024]);
			if(!defined($res2->{$cnode})) {
				return(0,"Error occured calling lvm_lvcreate - aborting.");
			} 
			if($res2->{$cnode} !~ /^OK/) {
				return(0,"Unable to create meta volume on '$cnode':\n$res2->{$cnode}");
			}
			$reqd2=undef;
		}
	}
	
	#################################################################
	# So meta will have been created, so perform database updates	#
	# finally. Note that the 'topology' field is taken from the	#
	# extras argument. It is expected lha_buildapp will have 	#
	# validated it is a valid value.				#
	#################################################################
	$sql="INSERT OR REPLACE INTO storage_drbd8 (devnum,appname,devport,vglv,topology,synched,validhost,vischeck,syncrate) VALUES ('$usedev','$appname','$inet_port','$vgname/$lvname','$extras{topology}','N','$hostname','','$syncrate');";
	
	$results=$reqd->run(cmd => ["perform_adhoc_sql","SQL",$sql,"DBTYPE","config"]);
	if(!defined($results)) {
		return(0,"No results returned from perform_adhoc_sql request.");
	}

	for $cnode (keys %$results) {
		if($results->{$cnode} !~ /^OK/) {
			return(0,"Failed to insert record into database on '$cnode' - error:\n$results->{$cnode}");
		}
	}

	#################################################################
	# If we have any additional arguments, so update the records	#
	# now to take account of them.					#
	#################################################################

	for my $xx ("auth_crc","data_crc","syncrate") {
		if(exists($extras{$xx})) {
			$sql="UPDATE storage_drbd8 SET $xx = '$extras{$xx}' WHERE devnum = '$usedev';";
			::msg("Setting '$xx' to '$extras{$xx}' for device '$usedev'.");

			$results=$reqd->run(cmd => ["perform_adhoc_sql","SQL",$sql,"DBTYPE","config"]);
			if(!defined($results)) {
				::warning("No results returned from perform_adhoc_sql request.");
			}

			for $cnode (keys %$results) {
				if($results->{$cnode} !~ /^OK/) {
					::warning("Failed to update record into database on '$cnode' - error:\n$results->{$cnode}");
				}
			}
		}
	}

	#################################################################
	# Now do the same but for ports table.				#
	#################################################################

	$sql="INSERT OR REPLACE INTO ports (number,type,detail) VALUES ('$inet_port','drbd','$vgname/$lvname');";
	
	$results=$reqd->run(cmd => ["perform_adhoc_sql","SQL",$sql,"DBTYPE","config"]);
	if(!defined($results)) {
		return(0,"No results returned from perform_adhoc_sql request.");
	}

	for $cnode (keys %$results) {
		if($results->{$cnode} !~ /^OK/) {
			return(0,"Failed to insert record into database on '$cnode' - error:\n$results->{$cnode}");
		}
	}
	return(1,undef);
}

#########################################################################
# Will return (1,undef) - if resources removed, or (0,errmsg) if not. 	#
# To remove a resource the storage_drbd8 and ports table both need	#
# updating to remove the references. This can be done on both hosts	#
# unlike the allocation above, which makes things easier.		#
#########################################################################

sub remove_resources($$$$$$) {
my ($self,$vgname,$lvname,$appname,$nodelist,$openlocally)=@_;

	#################################################################
	# Firstly remove the entry from the storage_drbd8 table, if	#
	# if exists...							#
	#################################################################
	my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($db)) {
		return(0,"Unable to open config local database connection.");
	}
	my $devport=undef;
	my $ref=$db->get_rows("SELECT * FROM storage_drbd8 WHERE vglv = '$vgname/$lvname';");
	if(!defined($ref) || !exists($ref->[0]->{devport})) {
		$devport=$ref->[0]->{devport};
	}
	$db->disconnect; $db=undef;

	my $clreqdxml=new clreqdxml("$::ENV{TRUECL_CFG}/clreqd.xml");
	if(! defined($clreqdxml)) {
		return (0,"Unable to load Request Daemon configuration info.", "Reason: " . clreqdxml::error());
	}

	my @N=split(/,/,$nodelist);
	my $reqd=new clreq(hosts => [@N], clreqdxml => $clreqdxml, timeout => 5);

	#################################################################
	# Firstly remove the entry from the storage_drbd8 table, if	#
	# if exists...							#
	#################################################################

	my $sql="DELETE FROM storage_drbd8 WHERE vglv = '$vgname/$lvname' AND appname = '$appname';";
	my $results=$reqd->run(cmd => ["perform_adhoc_sql","SQL",$sql,"DBTYPE","config"]);
	if(!defined($results)) {
		return(0,"No results returned from perform_adhoc_sql request.");
	}
	my $chost;
	for $chost (keys %$results) {
		if($results->{$chost} !~ /^OK/) {
			return(0,"Host '$chost' returned an error:\n$results->{$chost}");
		}
	}

	if(defined($devport)) {
		$sql="UPDATE OR REPLACE ports SET type = '',detail='' WHERE number = $devport;";
		my $results=$reqd->run(cmd => ["perform_adhoc_sql","SQL",$sql,"DBTYPE","config"]);
		for $chost (keys %$results) {
			if($results->{$chost} =~ /^OK/) {
				return(0,"Host '$chost' return an error:\n$results->{$chost}");
			}
		}
	}
	return(1,undef);
}

#########################################################################
# The device status can be used to indicate the status of the device on	#
# the specified host. For this storage spec it indicates whether it has	#
# been visisbility checked on the host or not before via the 		#
# storage_drbd8 table information.					#
#									#
# Returns (0,errmsg) - an error has occured.				#
#         (1,undef)  - validated the device is working as expected	#
#		       on the specified host.				#
#########################################################################

sub device_status($$$$$) {
my ($self,$vg,$lv,$appname,$host)=@_;

	return(1,undef);
}

sub local_devname($$$$) {
my ($self,$vg,$lv,$appname)=@_;


	my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($db)) {
		return(0,"Unable to open config local database connection.");
	}
	my $devnum;
	my $ref=$db->get_rows("SELECT * FROM storage_drbd8 WHERE vglv = '$vg/$lv';");
	if(defined($ref) || defined($ref->[0]->{devnum})) {
		$devnum=$ref->[0]->{devnum};
	} else {
		return(0,"Missing entry from storage_drbd8 table for vglv == '$vg/$lv'");
	}
	$db->disconnect; $db=undef;

	#################################################################
	# DRBD will always be on Linux, so will always have same	#
	# raw/block device settings...					#
	#################################################################

	if(-e "/dev/drbd/$devnum") {
		return(1,"/dev/drbd/$devnum","/dev/drbd/$devnum");
	}
	return(1,"/dev/drbd$devnum","/dev/drbd$devnum");
}

#########################################################################
# For the storage-spec docs the following must be returned by this	#
# function:								#
#									#
#		  (0,errmsg) - an error has occured.			#
#		  (2,[nodes]) - nodes not available.			#
#		  (1,[lvname,status,rate],...) where:			#
#			lvname = lvname					#
#			status = - 0==unsynched,1==synched,		#
#					2==sync_in_progress,		#
#					3==transitiioning		#
#########################################################################

sub sync_status($$$) {
my ($self,$vgname,$appname)=@_;

	my $h=hostname();
	#################################################################
	# First thing to do is to get a list of LV details from the	#
	# config database.						#
	#################################################################

	my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($db)) {
		$error_msg="Unable to open config local database connection.";
		return(0,$error_msg);
	}
	my $ref=$db->get_rows("SELECT * FROM storage_drbd8 WHERE vglv LIKE '$vgname/%';");
	my $ref2=$db->get_rows("SELECT * FROM applications WHERE name = '$appname';");
	if(!exists($ref->[0]->{vglv}) || !defined($ref->[0]->{vglv})) {
		$db->disconnect;$db=undef;
		return (1,undef);
	}

	my $onode;
	{
		my @NN=split(/,/,$ref2->[0]->{nodes});
		if(@NN!=2) {
			$error_msg="Unexpected node count found in application record.";
			return (0,$error_msg);
		}
		if(! grep {$_ eq $h} @NN) {
			$error_msg="None of the nodes listed for application '$appname' include the current node.";
			return (0,$error_msg);
		}

		if($NN[0] eq $h) {
			$onode=$NN[1];
		} else {
			$onode=$NN[0];
		}

	}

	#################################################################
	# Check if other node in cluster and return 2 if not.		#
	#################################################################
	my ($_x2,$_x3)=get_nodes_in_cluster();
	if($_x2==2 && ! grep {$_ eq $onode} @$_x3) {
		return(2,[$onode]);
	}

	#################################################################
	# Scan each line now to get the drbd dev details ... and to 	#
	# get status information.					#
	# If the device status has changed we also update it now.	#
	#################################################################

	my @results=();
	my $cref;

	my $clreqdxml=new clreqdxml("$::ENV{TRUECL_CFG}/clreqd.xml");
	if(! defined($clreqdxml)) {
		return (0,"Unable to load Request Daemon configuration info.", "Reason: " . clreqdxml::error());
	}
	my $reqd=new clreq(hosts => [$onode], clreqdxml => $clreqdxml, timeout => 5);

	for $cref (@$ref) {
		my $devnum=$cref->{devnum};
		my $devport=$cref->{devport};
		my $synched=$cref->{synched};
		my ($vg,$lv)=split(/\//,$cref->{vglv});
		my $vglv=$cref->{vglv};
		my $drbd=get_drbd_dev_status($cref->{devnum});
		if(!defined($drbd)) {
			::warning("Unable to ascertain status for device '$devnum' - not in /proc/drbd!!");
			next;
		}
		if($drbd->{cs} =~ /^sync/i) {
			push @results,[$lv,2,$drbd->{speed}];
			if($synched eq "Y") {
				$db->do("UPDATE storage_drbd8 SET synched = 'N' WHERE vglv = '$vglv';");
				my $sql="UPDATE storage_drbd8 SET synched = 'N' WHERE vglv = '$vglv';";
				my $results=$reqd->run(cmd => ["perform_adhoc_sql","SQL",$sql,"DBTYPE","config"]);
			}
			next;
		}
		if($drbd->{cs} eq "standalone" || $drbd->{ld} eq "inconsistent" || $drbd->{cs} eq "wfconnection") {
			push @results,[$lv,0,0];
			if($synched eq "Y") {
				$db->do("UPDATE storage_drbd8 SET synched = 'N' WHERE vglv = '$vglv';");
				my $sql="UPDATE storage_drbd8 SET synched = 'N' WHERE vglv = '$vglv';";
				my $results=$reqd->run(cmd => ["perform_adhoc_sql","SQL",$sql,"DBTYPE","config"]);
			}
			next;
		}
		if(!exists($drbd->{bad}) || $drbd->{bad}==0) {
			push @results,[$lv,1,1];
			if($synched ne "Y") {
				$db->do("UPDATE storage_drbd8 SET synched = 'Y' WHERE vglv = '$vglv';");
				my $sql="UPDATE storage_drbd8 SET synched = 'Y' WHERE vglv = '$vglv';";
				my $results=$reqd->run(cmd => ["perform_adhoc_sql","SQL",$sql,"DBTYPE","config"]);
			}
			next;
		}
		push @results,[$lv,3,0];
	}
	return (1,@results);
}

#########################################################################
# The drbd8 image type can be be out of sync obviously since each	#
# host has its own copy of the data. The number of times to attempt a 	#
# sync depends on the approach, but for the first version we will 	#
# just set it to 2, each which does the same thing initially.		#
#########################################################################

sub sync_attempts_supported($) {
my $self=$_[0];

	return 2;
}

#########################################################################
# The below is called via the syncd process when it attempts to get	#
# the specifed device to sync. This will be called on the node where	#
# the application is running - which may or may not have the good	#
# data copy - can be primary/stale for example if the local hardware	#
# has failed.								#
#									#
# At present we always attempt the same actions when we perform this	#
# check - in the future different attempt numbers might do different	#
# things.								#
#########################################################################

sub do_sync_attempt($$$$$) {
my ($self,$vgname,$lvname,$appname,$attempt)=@_;
my $h=hostname;

	#################################################################
	# First thing to do is to get a list of LV details from the	#
	# config database.						#
	#################################################################

	my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($db)) {
		$error_msg="Unable to open config local database connection.";
		return(0,$error_msg);
	}
	my $ref2=$db->get_rows("SELECT * FROM applications WHERE name = '$appname';");
	my $ref=$db->get_rows("SELECT * FROM storage_drbd8 WHERE vglv = '$vgname/$lvname';");
	$db->disconnect;$db=undef;
	if(!exists($ref->[0]->{vglv}) || !defined($ref->[0]->{vglv})) {
		$error_msg="Specified VG/LV '$vgname/$lvname' not found for storage_drbd8.";
		return (0,$error_msg);
	}
	my $devport=$ref->[0]->{devport};
	my $devnum=$ref->[0]->{devnum};
	if(!defined($ref2) || !defined($ref2->[0]->{nodes})) {
		$error_msg="Application missing node details";
		return (0,$error_msg);
	}
	my @NN=split(/,/,$ref2->[0]->{nodes});
	if(@NN!=2) {
		$error_msg="Unexpected node count found in application record.";
		return (0,$error_msg);
	}
	if(! grep {$_ eq $h} @NN) {
		$error_msg="None of the nodes listed for application '$appname' include the current node.";
		return (0,$error_msg);
	}

	my $onode;
	if($NN[0] eq $h) {
		$onode=$NN[1];
	} else {
		$onode=$NN[0];
	}

	#################################################################
	# We also need to check to see if the onode is actually up.	#
	#################################################################
	my ($_x2,$_x3)=get_nodes_in_cluster();
	if($_x2==2 && ! grep {$_ eq $onode} @$_x3) {
		return(0,"Other node for sync data is not running as yet.");
	}
	
	#################################################################
	# We are going to run some stuff on both of the nodes, so get 	#
	# the repl map details for both.				#
	#################################################################

	my $clreqdxml=new clreqdxml("$::ENV{TRUECL_CFG}/clreqd.xml");
	if(! defined($clreqdxml)) {
		return (0,"Unable to load Request Daemon configuration info.", "Reason: " . clreqdxml::error());
	}

	my $reqd=new clreq(hosts => [@NN], clreqdxml => $clreqdxml, timeout => 5);
	my $reqd2=new clreq(hosts => [$onode], clreqdxml => $clreqdxml, timeout => 5);

	my $rep_res=$reqd->run(cmd => ["ss_repl_ip","APPLICATION",$appname]);
	my $errs=0;
	my @RR=();
	for my $cnode (@NN) {
		if(! exists($rep_res->{$cnode})) {
			::error("ss_repl_ip did not get any results from node '$cnode'.");
			$errs++; next;
		}
		if($rep_res->{$cnode} !~ /^OK\s+/) {
			::error("Invalid result from node '$cnode' for ss_repl_ip:");
			::error($rep_res->{$cnode});
			$errs++; next;
		}
		my @F=split(/\s+/,$rep_res->{$cnode});
		push @RR,"$cnode:$F[1]";
	}
	if($errs) {
		exit(1);
	}

	#################################################################
	# We attempt to set up the IP details, just incase the other	#
	# node was not up before...					#
	#################################################################
	{
		my $cip; my $local_cip;
		my ($a1,$a2)=split(/:/,$RR[0]);
		my ($b1,$b2)=split(/:/,$RR[0]);
		if($a1 eq $h) {
			my @FF=split(/=/,$a2);
			$local_cip=$FF[1];
			@FF=split(/=/,$b2);
			$cip=$FF[1];
		} else {
			my @FF=split(/=/,$a2);
			$cip=$FF[1];
			@FF=split(/=/,$b2);
			$local_cip=$FF[1];
		}
		my $cmd=$self->{drbd_setup} . " /dev/drbd$devnum net $local_cip:$devport $cip:$devport C -m";
		system("$cmd >/dev/null 2>/dev/null");
	}

	#################################################################
	# @RR now contains the repl_map - so run the before start 	#
	# on the other node.						#
	#################################################################
	$results=$reqd2->run(cmd => ["ss_run_before_start_on_non_current","APPLICATION",$appname,"REPL_MAP",join(" ",@RR)]);
	my $abort=0;
	if(defined($results)) {
		for (keys(%$results)) {
			if($results->{$_} ne "OK") {
				::error("$_ : $results->{$_}");
				$abort=1;
			}
		}
	}
	if($abort) {
		$error_msg="Errors found in ss_run_before_start_on_non_current on '$onode'.";
		return(0,$error_msg);
	}

	#################################################################
	# Now the before script has been run, run the after script as	#
	# well - same arguments.					#
	#################################################################

	$results=$reqd2->run(cmd => ["ss_run_after_start_on_non_current","APPLICATION",$appname,"REPL_MAP",join(" ",@RR)]);
	$abort=0;
	if(defined($results)) {
		for (keys(%$results)) {
			if($results->{$_} ne "OK") {
				::error("$_ : $results->{$_}");
				$abort=1;
			}
		}
	}
	if($abort) {
		$error_msg="Errors found in ss_run_after_start_on_non_current on '$onode'.";
		return(0,$error_msg);
	}
	#################################################################
	# For the dummy configuration we assume that DRBD will handle	#
	# the reconnection automatically...				#
	#################################################################
	return (2,undef);

	#################################################################
	# Other stuff we might need, though we don't need 'em yet.	#
	#################################################################

	# my $devnum=$ref->[0]->{devnum};
	# my $devport=$ref->[0]->{devport};
	# my $topology=$ref->[0]->{topology};
	# return (2,undef);
}

#########################################################################
# Indicates that the device in question is currently synched. Can be a 	#
# no-op if necessary, or can update local info - in this case the	#
# storage_drbd8 table in the config database.				#
# (0,errmsg) - error occurred						#
# (1,undef) - OK							#
#########################################################################

sub register_synched($$$$) {
my ($self,$vgname,$lvname,$appname)=@_;

	my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($db)) {
		$error_msg="Unable to open config local database connection.";
		return(0,$error_msg);
	}
	my $ref=$db->do("UPDATE storage_drbd8 SET synched = '1' WHERE vglv = '$vgname/$lvname';");
	$db->disconnect;$db=undef;
	return(1,undef);
}

sub register_unsynched($$$$) {
my ($self,$vgname,$lvname,$appname)=@_;

	my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($db)) {
		$error_msg="Unable to open config local database connection.";
		return(0,$error_msg);
	}
	my $ref=$db->do("UPDATE storage_drbd8 SET synched = '0' WHERE vglv = '$vgname/$lvname';");
	$db->disconnect;$db=undef;
	return(1,undef);
}

#################################################################################
# The DRBD8 resize is only upwards at present unfortunately. Hence we need to	#
# check the old and newsize and abort with an error in these circumstances.	#
#################################################################################

sub resize_lv($$$$$$$) {
my  ($self,$vg,$lv,$appname,$running,$nodelist,$oldsize,$newsize)=@_;
my @other=();
my $h=hostname;

	my $f=0;
	for (split(/,/,$nodelist)) {
		push @other,$_ if $_ ne $h;
		$f=1 if $_ eq $h;
	}

	#################################################################
	# This action must be performed on a node that is suitable	#
	# for hosting the application in question and hence if that is	#
	# not the case then abort with an error now.			#
	#################################################################

	if($f==0) {
		return(6,"Node '$h' is not configured to host application '$appname'.");
	}

	#################################################################
	# Although DRBD8.2. supports shrinking, we do not support it	#
	# yet since no Linux file systems support it anyway!		#
	#################################################################

	if($newsize < $oldsize) {
		return(6,"DRBD 8.2.x can only grow replicated storage.");
	}

	#################################################################
	# Also DRBD 8.2.x can only resize if the application is		#
	# actually running in our case!					#
	#################################################################

	if(!$running) {
		return(6,"DRBD 8.2.x can only grow file systems if application is running.");
	}

	#################################################################
	# Now get hold of the DRBD device number...			#
	#################################################################
	my $cldb=new clstatdb("$::ENV{TRUECL_CFG}/config.db");
	my $cmd3="SELECT * FROM storage_drbd8 WHERE appname = '$appname' AND vglv = '$vg/$lv';";
	my $r=$cldb->get_rows($cmd3);
	$cldb->disconnect;
	$cldb=undef;
	
	if(!exists($r->[0]->{vglv}) || !defined($r->[0]->{vglv})) {
		$error_msg="Specified VG/LV '$vgname/$lvname' not found for storage_drbd8.";
		return (0,$error_msg);
	}

	my $devnum=$r->[0]->{devnum};

	#################################################################
	# We need to work out which node the application is running	#
	# on.								#
	# 								#
	# The resize function is only supported when the cluster is	# 
	# up and running - whether the application is running or not.	#
	#################################################################

	my $clreqdxml=new clreqdxml("$::ENV{TRUECL_CFG}/clreqd.xml");
	if(! defined($clreqdxml)) {
		return (0,"Unable to load Request Daemon configuration info.", "Reason: " . clreqdxml::error());
	}

	my $reqd=new clreq(hosts => [$h], clreqdxml => $clreqdxml, timeout => 5);
	my $results=$reqd->run(cmd => ["get_master_node"]);

	my $running_node=undef;
	my $master=undef;
	my $clup=0;
	if($results->{$h}=~/^ERROR No local cluster daemon/) {
		return(5,"Node '$h' is not part of a running cluster.");
	}
	if($results->{$h}=~/^OK\s+([a-zA-Z0-9_]+)/) {
		$master=$1;
	} else {
		return(5,"Unexpected result from 'get_master_node' query.");
	}

	#########################################################
	# Since we are running in the cluster we can query	#
	# the local status database to check to see if the	#
	# application in question is currently running.		#
	#########################################################

	my $clstatdb=new clstatdb("$::ENV{TRUECL_CFG}/status.db");
	if(!defined($clstatdb)) {
		return(0,"Unable to connect to 'status' database.\nReason: " . (clstatdb::error)[1]);
	}

	$r=$clstatdb->get_rows("SELECT * FROM app_status WHERE name = '$appname';");
	$clstatdb->disconnect;
	$clstatdb=undef;
	if(!defined($r) || !defined($r->[0]->{status})) {
		return(5,"Status database on node '$h' missing entry for application '$appname'.");
	}
	if($r->[0]->{status} ne "STOPPED") {
		$running_node=$r->[0]->{host};
	}

	#########################################################################
	# On both nodes we need to ensure the logical volume size is		#
	# increased if necessary.						#
	#########################################################################
	my $reqd2=new clreq(hosts => [split(/,/,$nodelist)], clreqdxml => $clreqdxml, timeout => 5);
	::msg("Running: lvm_lvresize VGNAME $vg LVNAME $lv KBSIZE $newsize");
	$r=$reqd2->run(cmd => ["lvm_lvresize","VGNAME",$vg,"LVNAME",$lv,"KBSIZE",$newsize], timeout => 7200);

	#########################################################################
	# We need to check the results are OK across all nodes.			#
	# And ensure the new size is the same, and if not report on the		#
	# smallest.								#
	#########################################################################

	my $e=0;
	my $newsize2=undef;
	my @hh=();
	::debug("lvm_lvresize settings:");
	::debug(Dumper($r));
	for my $chost (keys %$r) {
		if($r->{$chost} !~ /^OK/) {
			::msg("Node '$chost' did not resize LV as expected:");
			::msg($r->{$chost});
			$e++;
			push @hh,$chost;
		}
		my ($x)=($r->{$chost} =~ /^OK\s+(\d+)/);
		if(!defined($newsize2)) {
			$newsize2=$x;
		} elsif($x < $newsize2) {
			$newsize2=$x;
		}
	}

	#########################################################################
	# If not all hosts have performed the resize then above before		#
	# going any further.							#
	#########################################################################
	if($e) {
		return(6,"The following hosts did not successfully perform resize: " . join(",",@hh));
	}

	#########################################################################
	# Now the logical volume has been resized we need to perform the	#
	# action on the node in question. We do this using the custom		#
	# 'drbd8_grow LVNAME n VGNAME n DRBDSETUP bin DEVNUM n' routine.	#
	#########################################################################

	$reqd2=new clreq(hosts => [$running_node], clreqdxml => $clreqdxml, timeout => 5);
	::msg("Performing DRBD8 resize.");
	::msg("Running: drbd8_grow VGNAME $vg LVNAME $lv DRBDSETUP $self->{drbd_setup} DEVNUM $devnum APPNAME $appname");
	$r=$reqd2->run(cmd => ["drbd8_grow","VGNAME",$vg,"LVNAME",$lv,"DRBDSETUP",$self->{drbd_setup},"DEVNUM",$devnum,"APPNAME",$appname], timeout => 60);
	if($r->{$running_node} !~ /^OK\s+(\d+)/) {
		return(6,"Failed to grow replicated LV: " . $r->{$running_node});
	}
	my $ppp=$1;

	#########################################################################
	# The replicated storage is underway via the pid ppp - so we need to	#
	# hand around until complete.						#
	#########################################################################

	while(1) {
		select(undef,undef,undef,10);
		$r=$reqd2->run(cmd => ["query_pid","PID","$ppp"], timeout => 10);
		if($r->{$running_node} !~ /^(COMPLETE|RUNNING)/) {
			return(6,"Failed to grow replicated LV: " . $r->{$running_node});
		}
		if($r->{$running_node} =~ /^COMPLETE/) {
			last;
		}
		::msg("Waiting for process '$ppp' to complete...");
	}
	return (0,$newsize2);
}

#################################################################################
# This validate routine is called after the logical volumes across the nodes	#
# in question have been validated. hence it is responsible for ensuring that	#
# all necessary resources have been allocated locally and aborting if not...	#
#################################################################################

sub validate {
my ($self,$application,$vg,$lv,$nlist,$running_on_node)=@_;

	my $clreqdxml=new clreqdxml("$::ENV{TRUECL_CFG}/clreqd.xml");
	if(! defined($clreqdxml)) {
		return (1,"Unable to load Request Daemon configuration info.", "Reason: " . clreqdxml::error());
	}
	my @N=split(/,/,$nlist);
	my $req=new clreq(hosts => [@N], clreqdxml => $clreqdxml, timeout => 5);
	if(!defined($req)) {
		return(1,"Unable to create request object - aborting.");
	}
	my $cmd="SELECT * FROM storage_drbd8 WHERE appname = '$application' AND vglv = '$vg/$lv';";
	my $results=$req->run(cmd => ["perform_adhoc_sql","DBTYPE","config","SQL",$cmd]);

	my $h=hostname;
	$r=_map_select_output($results);
	my $rec;
	for $rec (@{$r->{$h}}) {
		if(!ref($rec)) {
			return(1,"Missing record details for '$vg/$lv' in storage_drbd8");
		}
	}
	return(0,undef);
}

sub min_sync_interval {
	return 10;
}

sub sync_boundaries {
	return (10,100000);
}

#########################################################################
# If the lvname = "$$ALL" then all lv's for the application will be	#
# set to the specified rate.						#
#########################################################################

sub set_sync_rate($$$$$;$) {
my ($self,$vgname,$lvname,$appname,$rate,$perm)=@_;

	my $h=hostname();
	$perm=0 if !defined($perm);
	#################################################################
	# Map the vg/lv to a device number...				#
	#################################################################

	my $cldb=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($cldb)) {
		return(1,"Unable to connect to 'config' database.\nReason: " . (cldb::error)[1]);
	}
	my $r=$cldb->exists("table","storage_drbd8");
	return(1,"Table 'storage_drbd8' does not exist.") if !$r;

	my $clstatdb=new clstatdb("$::ENV{TRUECL_CFG}/status.db");
	my $ref3=$clstatdb->get_rows("SELECT * FROM app_status WHERE name = '$appname';");
	my $running=0;
	if(defined($ref3) && defined($ref3->[0]->{status}) &&
		$ref3->[0]->{status} eq "RUNNING" && $ref3->[0]->{host} eq $h) {
		$running=1;
	}

	#################################################################	
	# If the table does not exist, then look for the device		#
	# details.							#
	#################################################################	
 
 	my $ref;
	my $ref2;
	if($perm) {
		$ref2=$cldb->get_rows("SELECT * FROM applications WHERE name = '$appname';");
		if($lvname eq '$$ALL') {
			$cldb->do("UPDATE storage_drbd8 SET syncrate = '$rate' WHERE vglv LIKE '$vgname/%';");
		} else {
			$cldb->do("UPDATE storage_drbd8 SET syncrate = '$rate' WHERE vglv = '$vgname/$lvname';");
		}
	}
	if($lvname eq '$$ALL') {
		$ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv LIKE '$vgname/%';");
	} else {
		$ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv = '$vgname/$lvname';");
	}
	$cldb->disconnect();
	if(!defined($ref) || !defined($ref->[0]->{devnum})) {
		return(1,"Device details for '$vgname/$lvname' not found.");
	}
	#################################################################
	# If perm then we need to update the local and remote copies.	#
	#################################################################
	if($perm) {
		my $onode;
		{
			my @NN=split(/,/,$ref2->[0]->{nodes});
			if(@NN!=2) {
				$error_msg="Unexpected node count found in application record.";
				return (1,$error_msg);
			}
			if(! grep {$_ eq $h} @NN) {
				$error_msg="None of the nodes listed for application '$appname' include the current node.";
				return (1,$error_msg);
			}

			if($NN[0] eq $h) {
				$onode=$NN[1];
			} else {
				$onode=$NN[0];
			}
		}
		my $clreqdxml=new clreqdxml("$::ENV{TRUECL_CFG}/clreqd.xml");
		if(! defined($clreqdxml)) {
			return (0,"Unable to load Request Daemon configuration info.", "Reason: " . clreqdxml::error());
		}
		my $reqd=new clreq(hosts => [$onode], clreqdxml => $clreqdxml, timeout => 5);
		my $sql;
		if($lvname eq '$$ALL') {
			$sql="UPDATE storage_drbd8 SET syncrate = '$rate' WHERE vglv LIKE '$vgname/%';";
		} else {
			$sq="UPDATE storage_drbd8 SET syncrate = '$rate' WHERE vglv = '$vgname/$lvname';";
		}
		my $results=$reqd->run(cmd => ["perform_adhoc_sql","SQL",$sql,"DBTYPE","config"]);
	}
	if($running) {
		my $drbd_setup=$self->{drbd_setup};
		for my $crec (@$ref) {
			my $devnum=$crec->{devnum};
			my $cmd=$self->{drbd_setup} . " /dev/drbd$devnum syncer --rate $rate";
			local $SIG{CHLD}='IGNORE';
			my $rc=`$cmd 2>&1 >/dev/null`;
			chomp $rc;
			return(1,"Unable to set rate for '$vgname/$lvname' to '$rate' for device '/dev/drbd$devnum'.[error: $rc]") if length($rc);
		}
	}
	return(0,undef);
}

sub supported_os {
	return(1) if $Config{osname} eq "linux";
	return(0);
}

#################################################################
# drbd has no pre-checks for the node to validate/import on.	#
#################################################################

sub check_use_state($$$$$) {
my ($self,$vgname,$appname,$action,$nodelist)=@_;

	return(0,undef);
}

#################################################################
# Live migration routines not available [this is because drbd	#
# 0.7.x does not support dual primaries.			#
#################################################################

sub support_lmigrate($) {
	return 1;
}

#################################################################
# This is the first routine called as part of a live migration.	#
# It, in this instance, does nothing to the storage.		#
#################################################################

sub onode_premigrate($$$$) {
	return(0,undef);
}

#################################################################
# For drbd8 before the node can be migrated the storage 	#
# must be modified to make it primary too...			#
#################################################################

sub nnode_premigrate($$$$) {
my ($self,$vgname,$appname,$onode)=@_;

	# Get a list of volumes we currently support...		#

	my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($db)) {
		my $error_msg="Unable to open config local database connection.";
		return(1,$error_msg);
	}
	my $ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv LIKE '$vgname/%';");
	my @un=();
	for my $cref (@$ref) {
		next if !defined($cref->{synched});
		my ($lv)=($cref->{vglv} =~ /.*\/(.*)/);
		if(!defined($lv)) {
			return(1,"Unable to find valid data in storage_drbd8.");
		}
		push @un,$lv if (to_bool($cref->{synched})==0);
	}
	if(@un) {
		return(1,"Unable to configure for premigrate because the following logical volumes in volume group '$vgname' are not synchronised: " . join(",",@un));
	}

	#################################################################
	# Using the device information we make these devices primary	#
	# now.								#
	#################################################################

	my $failed=undef;
	for my $crec (@$ref) {
		my $devnum=$crec->{devnum};
		my $cmd="$self->{drbd_setup}=$drbd_setup /dev/drbd$devnum primary";
		local $SIG{CHLD}='IGNORE';
		my $rc=`$cmd 2>&1 >/dev/null`;
		chomp $rc;
		if(length($rc)) {
			$failed=$crec;
			last;
		}
	}
	if(defined($failed)) {
		for my $crec (@$ref) {
			my $devnum=$crec->{devnum};
			my $cmd="$self->{drbd_setup}=$drbd_setup /dev/drbd$devnum secondary";
			local $SIG{CHLD}='IGNORE';
			system("$cmd 2>/dev/null >/dev/null");
		}
		return(1,"Failed to make DRBD device '/dev/drbd$devnum' primary.");
	}
	return(0,undef);
}

#################################################################
# This is called once the migration has been successful. It 	#
# stops access to the older devices by marking them as 		#
# secondary for now...						#
#################################################################

sub onode_postmigrate($$$$) {
my ($self,$vgname,$appname,$onode)=@_;

	# Get a list of volumes we currently support...		#

	my $db=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($db)) {
		my $error_msg="Unable to open config local database connection.";
		return(1,$error_msg);
	}
	my $ref=$cldb->get_rows("SELECT * FROM storage_drbd8 WHERE vglv LIKE '$vgname/%';");

	#################################################################
	# Using the device information we make these devices primary	#
	# now.								#
	#################################################################

	my $failed=undef;
	for my $crec (@$ref) {
		my $devnum=$crec->{devnum};
		my $cmd="$self->{drbd_setup}=$drbd_setup /dev/drbd$devnum secondary";
		local $SIG{CHLD}='IGNORE';
		my $rc=`$cmd 2>&1 >/dev/null`;
		chomp $rc;
		if(length($rc)) {
			$failed=$crec;
			last;
		}
	}
	my @un=();
	for my $crec (@$ref) {
		my $devnum=$crec->{devnum};
		my $cmd="$self->{drbd_setup}=$drbd_setup /dev/drbd$devnum secondary";
		local $SIG{CHLD}='IGNORE';
		my $rc=`$cmd 2>&1 >/dev/null`;
		if(length($rc)) {
			push @un,"/dev/drbd$devnum";
		}
	}
	if(@un) {
		return(1,"Failed to move devices back to secondary: " . join(",",@un));
	}
	return(0,undef);
}

#################################################################
# For drbd8 we do nothing on the new node once everything is	#
# running, so just return OK.					#
#################################################################

sub nnode_postmigrate($$$$) {
	return(0,undef);
}

sub strfmt_logical_short($$) {
my $self=shift;
my $dev=shift;

	return $dev;
}

sub strfmt_logical_long($$) {
my $self=shift;
my $dev=shift;

	return $dev;
}
1;

